
import pandas as pd
import seaborn as sns
import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
import os
import librosa
import librosa.display
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.patches as mpatches


df = pd.read_csv("marketing_campaign.csv").drop(columns= ["Unnamed: 0"])


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 2240 entries, 0 to 2239
Data columns (total 19 columns):
 #   Column             Non-Null Count  Dtype  
---  ------             --------------  -----  
 0   ID                 2240 non-null   int64  
 1   Year_Birth         2240 non-null   int64  
 2   Education          2240 non-null   object 
 3   Marital_Status     2240 non-null   object 
 4   Income             2017 non-null   float64
 5   Kidhome            2240 non-null   int64  
 6   Teenhome           2240 non-null   int64  
 7   Dt_Customer        2240 non-null   object 
 8   Recency            2240 non-null   int64  
 9   MntCoffee          2035 non-null   float64
 10  MntFruits          2240 non-null   int64  
 11  MntMeatProducts    2240 non-null   int64  
 12  MntFishProducts    2240 non-null   int64  
 13  MntSweetProducts   2240 non-null   int64  
 14  MntGoldProds       2227 non-null   float64
 15  NumWebVisitsMonth  2040 non-null   float64
 16  Complain           2240 non-null   int64  
 17  NumPurchases       2240 non-null   int64  
 18  UsedCampaignOffer  2240 non-null   int64  
dtypes: float64(4), int64(12), object(3)
memory usage: 332.6+ KB


df.describe()


df.isnull().sum()

ID                     0
Year_Birth             0
Education              0
Marital_Status         0
Income               223
Kidhome                0
Teenhome               0
Dt_Customer            0
Recency                0
MntCoffee            205
MntFruits              0
MntMeatProducts        0
MntFishProducts        0
MntSweetProducts       0
MntGoldProds          13
NumWebVisitsMonth    200
Complain               0
NumPurchases           0
UsedCampaignOffer      0
dtype: int64


(df.isnull().sum() / len(df)) * 100

ID                   0.000000
Year_Birth           0.000000
Education            0.000000
Marital_Status       0.000000
Income               9.955357
Kidhome              0.000000
Teenhome             0.000000
Dt_Customer          0.000000
Recency              0.000000
MntCoffee            9.151786
MntFruits            0.000000
MntMeatProducts      0.000000
MntFishProducts      0.000000
MntSweetProducts     0.000000
MntGoldProds         0.580357
NumWebVisitsMonth    8.928571
Complain             0.000000
NumPurchases         0.000000
UsedCampaignOffer    0.000000
dtype: float64


cor = df.corr(numeric_only=True)
plt.figure(figsize=(15,15))
sns.heatmap(cor, annot=True, center=0 , fmt=".3f" , cmap = "rocket" , linewidth=0.75)
plt.title('Correlation')
plt.show()


nu = df.nunique().reset_index()
nu


numerical_features = [ "Year_Birth" , "Income" , "Kidhome" , "Teenhome" , "Recency" , "MntCoffee" , "MntFruits" , "MntMeatProducts" , "MntFishProducts" , "MntSweetProducts" , "NumWebVisitsMonth" ]
categorical_features = ["ID" , "Education" , "Marital_Status" , "Complain" , "UsedCampaignOffer" ,"Dt_Customer" ]


plt.figure(figsize=(10,5))
plt.title("number of unique vals for every f ")
plt.xticks(rotation = 80)
nu.columns = ['feature','num of unique vals']
ax = sns.barplot(x='feature', y='num of unique vals', data=nu)


fig, axes = plt.subplots( nrows=len(df.columns), ncols=1, figsize=(7, 4*len(df.columns)))
for i, col in enumerate(df.columns):

    axes[i].scatter( df[col] , df["NumPurchases"] , c="blue", alpha=0.5, marker=r'$\clubsuit$',
           label="Luck")
    axes[i].set_title(col + ' to NumPurchases' + " relation")
    axes[i].set_xlabel(col)
    axes[i].set_ylabel("NumPurchases")

plt.tight_layout()

plt.show()


df.plot(kind="hexbin",y="NumPurchases", x="MntCoffee",gridsize=20)

<Axes: xlabel='MntCoffee', ylabel='NumPurchases'>


numerical_cols = df.select_dtypes(include=['number']).columns.tolist()

print(f"Numerical columns: {numerical_cols}")

fig, axes = plt.subplots(nrows=len(numerical_cols), ncols=1, figsize=(6, 6*len(numerical_cols)))

for i, col in enumerate(numerical_cols):
    axes[i].hexbin(df[col], df["NumPurchases"], gridsize=30, cmap='Greens')
    axes[i].set_title(f"Hexbin of {col} vs NumPurchases")
    axes[i].set_xlabel(col)
    axes[i].set_ylabel("NumPurchases")

plt.tight_layout()
plt.show()

Numerical columns: ['ID', 'Year_Birth', 'Income', 'Kidhome', 'Teenhome', 'Recency', 'MntCoffee', 'MntFruits', 'MntMeatProducts', 'MntFishProducts', 'MntSweetProducts', 'MntGoldProds', 'NumWebVisitsMonth', 'Complain', 'NumPurchases', 'UsedCampaignOffer']


for column in df.columns:
    # Obtain the value counts for the current feature
    value_counts = df[column].value_counts()
    if (len(value_counts) < 50):
        plt.figure(figsize=(10, 5)) 
        value_counts.plot(kind='bar')
        plt.title(f'Number of observations for each unique value of {column}')
        plt.xlabel('Unique Values')
        plt.ylabel('Number of Observations')
        plt.show()
    elif df[column].dtype == 'int64' or df[column].dtype == 'float64':
        plt.figure(figsize=(10, 5))  # Adjust the figure size as needed
        df[column].plot(kind='hist', bins=30)  # You can change the number of bins as needed
        plt.title(f'Distribution of {column}')
        plt.xlabel(column)
        plt.ylabel('Frequency')
        plt.show()


median = df['Income'].median()
mean = df['Income'].mean()
std = df['Income'].std()
min_ = df["Income"].min()
max_ = df["Income"].max()

print(f"Income Median: {median:.2f}")
print(f"Income Mean: {mean:.2f}")
print(f"Income Standard Deviation: {std:.2f}")
print(f"Income min: {min_:.2f}")
print(f"Income max: {max_:.2f}")

Income Median: 51369.00
Income Mean: 52297.08
Income Standard Deviation: 25543.11
Income min: 2447.00
Income max: 666666.00


df['Income'].fillna(df['Income'].mean(), inplace=True)


median = df['MntCoffee'].median()
mean = df['MntCoffee'].mean()
std = df['MntCoffee'].std()
min_ = df["MntCoffee"].min()
max_ = df["MntCoffee"].max()

print(f"MntCoffee Median: {median:.2f}")
print(f"MntCoffee Mean: {mean:.2f}")
print(f"MntCoffee Standard Deviation: {std:.2f}")
print(f"MntCoffee min: {min_:.2f}")
print(f"MntCoffee max: {max_:.2f}")

MntCoffee Median: 177.00
MntCoffee Mean: 304.24
MntCoffee Standard Deviation: 337.52
MntCoffee min: 0.00
MntCoffee max: 1493.00


df['MntCoffee'].fillna(df['MntCoffee'].median(), inplace=True)


median = df['MntGoldProds'].median()
mean = df['MntGoldProds'].mean()
std = df['MntGoldProds'].std()
min_ = df["MntGoldProds"].min()
max_ = df["MntGoldProds"].max()

print(f"MntGoldProds Median: {median:.2f}")
print(f"MntGoldProds Mean: {mean:.2f}")
print(f"MntGoldProds Standard Deviation: {std:.2f}")
print(f"MntGoldProds min: {min_:.2f}")
print(f"MntGoldProds max: {max_:.2f}")

MntGoldProds Median: 24.00
MntGoldProds Mean: 43.85
MntGoldProds Standard Deviation: 51.90
MntGoldProds min: 0.00
MntGoldProds max: 362.00


df.dropna(subset=['MntGoldProds'], inplace=True)


median = df['NumWebVisitsMonth'].median()
mean = df['NumWebVisitsMonth'].mean()
std = df['NumWebVisitsMonth'].std()
min_ = df["NumWebVisitsMonth"].min()
max_ = df["NumWebVisitsMonth"].max()

print(f"NumWebVisitsMonth Median: {median:.2f}")
print(f"NumWebVisitsMonth Mean: {mean:.2f}")
print(f"NumWebVisitsMonth Standard Deviation: {std:.2f}")
print(f"NumWebVisitsMonth min: {min_:.2f}")
print(f"NumWebVisitsMonth max: {max_:.2f}")

NumWebVisitsMonth Median: 6.00
NumWebVisitsMonth Mean: 5.33
NumWebVisitsMonth Standard Deviation: 2.44
NumWebVisitsMonth min: 0.00
NumWebVisitsMonth max: 20.00


df['NumWebVisitsMonth'].fillna(df['NumWebVisitsMonth'].mode()[0], inplace=True)


df = df[df["Kidhome"] >= 0]


df.isnull().sum()

ID                   0
Year_Birth           0
Education            0
Marital_Status       0
Income               0
Kidhome              0
Teenhome             0
Dt_Customer          0
Recency              0
MntCoffee            0
MntFruits            0
MntMeatProducts      0
MntFishProducts      0
MntSweetProducts     0
MntGoldProds         0
NumWebVisitsMonth    0
Complain             0
NumPurchases         0
UsedCampaignOffer    0
dtype: int64


from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
temp = pd.DataFrame(scaler.fit_transform(df[numerical_features]), columns=numerical_features, index=df.index) 
df = pd.concat([temp , df[categorical_features] , df["NumPurchases"]] , axis = 1 )


categorical_ordinal_features = ["Education" , "Dt_Customer"]
categorical_nominal_features = ["Marital_Status" ]


from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()
for column in categorical_ordinal_features:
    df[column] = encoder.fit_transform(df[column])


from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer

# Define a transformer for the one-hot encoded features
column_transformer = ColumnTransformer(
    transformers=[
        ('category_one_hot', OneHotEncoder(), categorical_nominal_features)
    ],
    remainder='passthrough'
)

# Fit the transformer and apply it to the dataframe
df_encoded = column_transformer.fit_transform(df)

# Get the feature names for the one-hot encoded columns
one_hot_features = column_transformer.named_transformers_['category_one_hot'].get_feature_names_out(categorical_nominal_features)

# Combine the new one-hot encoded column names with the remainder column names
new_column_names = one_hot_features.tolist() + df.drop(columns=categorical_nominal_features).columns.tolist()

# Convert the resulting encoded data back to a DataFrame
df = pd.DataFrame(df_encoded, index=df.index, columns=new_column_names)


df = df.drop(columns=['Recency'])


from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    df.drop(columns=["ID" , "NumPurchases"]),  # features
    df['NumPurchases'],                 # target
    test_size=0.2,               # 75/25 split, a quarter of data used for testing
    random_state=42              # Seed for random number generator for reproducibility
    #,stratify=df['NumPurchases']         # Stratify split based on labels, if classification problem
)


from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier()

kf = KFold(n_splits=5, random_state=42, shuffle=True)

# Split feature and target variables
X = df.drop('Complain', axis=1)
y = df['Complain']
scores = cross_val_score(model, X, y, cv=kf)

print(scores)
print(scores.mean())

[0.99101124 0.99101124 0.99325843 0.99099099 0.98648649]
0.9905516752707765


from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LinearRegression

model = LinearRegression()

kf = KFold(n_splits=5, random_state=42, shuffle=True)

X = df.drop('NumPurchases', axis=1)
y = df['NumPurchases']

scores = cross_val_score(model, X, y, scoring='neg_mean_squared_error', cv=kf)

# The scores are usually negative because cross_val_score uses utility functions (higher is better),
# rather than cost functions (lower is better). We can negate the scores to make them positive.
mse_scores = -scores

# Calculate RMSE for each fold
rmse_scores = np.sqrt(mse_scores)

print(rmse_scores)
print(rmse_scores.mean())

[4.92170743 5.11797588 4.81550073 4.79852867 4.79432831]
4.889608204325069


def linear_regression(input, output):
    sum_input = np.sum(input)
    sum_output = np.sum(output)
    mult_in_out = np.sum(input * output)
    SSi = np.sum(input ** 2)
    intercept = (mult_in_out - (len(input) * (sum_input / len(input)) * (sum_output / len(output)))) / (SSi - len(input) * ((sum_input / len(input)) ** 2))
    slope = (sum_output / len(output)) - intercept * (sum_input / len(input))
    print ("intercept: " , intercept)
    print ("slope: " , slope)
    return (intercept, slope)


def get_regression_predictions(input, intercept, slope):
    predicted_values = [(intercept * x) + slope for x in input]
    return (predicted_values)


intercept, slope = linear_regression(X_train['MntCoffee'], y_train)
y_pred = get_regression_predictions(X_test['MntCoffee'], intercept, slope)

intercept:  24.37756062027725
slope:  10.036584946790207


def get_root_mean_square_error(predicted_values, actual_values):
    if len(predicted_values) != len(actual_values):
        raise ValueError("The lengths of predicted and actual values must match.")
    residuals = np.subtract(predicted_values, actual_values)
    mean_squared_error = np.mean(np.square(residuals))
    root_mean_square_error = np.sqrt(mean_squared_error)
    
    return root_mean_square_error


def get_r2_score(predicted_values, actual_values):
    if len(predicted_values) != len(actual_values):
        raise ValueError("The lengths of predicted and actual values must match.")

    mean_actual_values = sum(actual_values) / len(actual_values)
    total_sum_of_squares = sum((y_i - mean_actual_values) ** 2 for y_i in actual_values)
    residual_sum_of_squares = sum((y_i - y_hat) ** 2 for y_i, y_hat in zip(actual_values, predicted_values))
    r_squared = 1 - (residual_sum_of_squares / total_sum_of_squares)
    
    return r_squared


for feature in X_train.columns:
    print(feature + ":")
    intercept, slope = linear_regression(X_train[feature], y_train)
    y_pred = get_regression_predictions(X_test[feature], intercept, slope)
    print ("R2 score: " , get_r2_score(y_pred, y_test))
    print ("RMSE score: " , get_root_mean_square_error(y_pred, y_test))
    print ("-------------------------------------------------------------")

Marital_Status_Absurd:
intercept:  4.658783783783784
slope:  14.841216216216216
R2 score:  -0.00010390650365543763
RMSE score:  7.641075728228752
-------------------------------------------------------------
Marital_Status_Alone:
intercept:  -1.5156807511737085
slope:  14.849014084507042
R2 score:  -8.414175034521243e-05
RMSE score:  7.641000223712682
-------------------------------------------------------------
Marital_Status_Divorced:
intercept:  0.08033522448311568
slope:  14.838143036386448
R2 score:  0.0003498419691880805
RMSE score:  7.639342148463077
-------------------------------------------------------------
Marital_Status_Married:
intercept:  0.3566357911326898
slope:  14.709057639524245
R2 score:  -0.002752863056369126
RMSE score:  7.651188423669955
-------------------------------------------------------------
Marital_Status_Single:
intercept:  -0.8057267759562826
slope:  15.01639344262295
R2 score:  -0.0005984190231074216
RMSE score:  7.642964602278193
-------------------------------------------------------------
Marital_Status_Together:
intercept:  -0.1968683717006929
slope:  14.897943640517898
R2 score:  -0.000466201445892267
RMSE score:  7.642459620645169
-------------------------------------------------------------
Marital_Status_Widow:
intercept:  2.164504849988719
slope:  14.77097902097902
R2 score:  -0.0006755037599268654
RMSE score:  7.643258998390148
-------------------------------------------------------------
Marital_Status_YOLO:
intercept:  4.158220720720721
slope:  14.84177927927928
R2 score:  -0.00010240956582352467
RMSE score:  7.641070009713139
-------------------------------------------------------------
Year_Birth:
intercept:  -10.672622288098568
slope:  22.692670403956413
R2 score:  0.03579658032957689
RMSE score:  7.502677446106312
-------------------------------------------------------------
Income:
intercept:  103.76636011632426
slope:  7.0465910211660825
R2 score:  0.391078641048637
RMSE score:  5.9622824508097345
-------------------------------------------------------------
Kidhome:
intercept:  -13.472629229375233
slope:  17.83953236535614
R2 score:  0.2624650764959007
RMSE score:  6.561803742087584
-------------------------------------------------------------
Teenhome:
intercept:  3.6851715380904193
slope:  13.912727908937157
R2 score:  0.020374086037364014
RMSE score:  7.562442307134854
-------------------------------------------------------------
MntCoffee:
intercept:  24.37756062027725
slope:  10.036584946790207
R2 score:  0.40742232801611344
RMSE score:  5.881723219602103
-------------------------------------------------------------
MntFruits:
intercept:  17.243768096360434
slope:  12.531510237980903
R2 score:  0.20978239682514876
RMSE score:  6.792119089476424
-------------------------------------------------------------
MntMeatProducts:
intercept:  31.558831006450994
slope:  11.78603055425027
R2 score:  0.3519023120588425
RMSE score:  6.15109136833164
-------------------------------------------------------------
MntFishProducts:
intercept:  17.06052812201244
slope:  12.357334598039422
R2 score:  0.2089581396455198
RMSE score:  6.7956605151290175
-------------------------------------------------------------
MntSweetProducts:
intercept:  23.316106222490653
slope:  12.46186285272812
R2 score:  0.21482853368025734
RMSE score:  6.770397949617261
-------------------------------------------------------------
NumWebVisitsMonth:
intercept:  -19.063216032546038
slope:  20.08616067216268
R2 score:  0.04906929654559966
RMSE score:  7.45085955150551
-------------------------------------------------------------
Education:
intercept:  0.6715343939853199
slope:  13.232205849328876
R2 score:  0.006862036574003083
RMSE score:  7.614418344241853
-------------------------------------------------------------
Complain:
intercept:  -2.695761098306443
slope:  14.872231686541737
R2 score:  -0.0035454774611001216
RMSE score:  7.65421172306039
-------------------------------------------------------------
UsedCampaignOffer:
intercept:  4.37565845978823
slope:  13.672559569561875
R2 score:  0.06361836218415307
RMSE score:  7.393641457926658
-------------------------------------------------------------
Dt_Customer:
intercept:  -0.0005871603497505455
slope:  15.03737891467046
R2 score:  -0.0009229187902186631
RMSE score:  7.644203830293161
-------------------------------------------------------------


def predict_output(feature_matrix, weights, bias):
    predictions = np.dot(feature_matrix, weights) + bias
    return predictions


def feature_derivative(errors, feature):
    derivative = 2 * np.dot(feature, errors)
    return derivative


def regression_gradient_descent(feature_matrix, outputs, weights, bias, step_size, tolerance):
    converged = False
    while not converged:
        predictions = predict_output(feature_matrix, weights, bias)
        errors = outputs - predictions

        gradient = -1 * feature_derivative(feature_matrix, errors)
        weights = weights - (step_size * gradient)

        bias_gradient = -2 * np.sum(errors)
        bias = bias - (step_size * bias_gradient)

        if np.linalg.norm(gradient) < tolerance:
            converged = True

    return weights, bias


# Utility functions for multiple regression

def normalize_features(chosen_features, data_frame):
    for feature in chosen_features:
        data_frame[feature] = (data_frame[feature] - data_frame[feature].mean()) / data_frame[feature].std()
    return data_frame


def n_feature_regression(chosen_feature_matrix, target_matrix, keywords):
    initial_weights = keywords['initial_weights']
    step_size = keywords['step_size']
    tolerance = keywords['tolerance']
    bias = keywords['bias']
    weights = np.array(initial_weights)
    weights, bias = regression_gradient_descent(chosen_feature_matrix, target_matrix, weights, bias, step_size,
                                                tolerance)

    return weights, bias

def get_weights_and_bias(chosen_features):

    keywords = {
        'initial_weights': np.array([.5]*len(chosen_features)),
        'step_size': 1.e-4,
        'tolerance': 1.e-10,
        'bias': 0
    }

    chosen_feature_dataframe = X_train[chosen_features]
    #chosen_feature_dataframe = normalize_features(chosen_features, chosen_feature_dataframe)
    chosen_feature_matrix = chosen_feature_dataframe.to_numpy()

    target_column = y_train
    target_matrix = target_column.to_numpy()
    
    train_weights, bias = n_feature_regression(chosen_feature_matrix, target_matrix, keywords)

    return chosen_feature_matrix, train_weights, bias


import pandas as pd
import numpy as np

X_train, X_test, y_train, y_test 

chosen_features = ["MntCoffee" , "Income"]

chosen_feature_matrix, train_weights, bias = get_weights_and_bias(chosen_features)


y_pred = predict_output(X_test[chosen_features], train_weights, bias)


print ("R2 score: " , get_r2_score(y_pred, y_test))
print ("RMSE score: " , get_root_mean_square_error(y_pred, y_test))

R2 score:  0.49407273237289595
RMSE score:  5.434705128367993


chosen_features = ["MntCoffee" , "MntMeatProducts" , "Income"]

chosen_feature_matrix, train_weights, bias = get_weights_and_bias(chosen_features)

y_pred = predict_output(X_test[chosen_features], train_weights, bias)

print ("R2 score: " , get_r2_score(y_pred, y_test))
print ("RMSE score: " , get_root_mean_square_error(y_pred, y_test))

R2 score:  0.5264687263939163
RMSE score:  5.257826794516655


chosen_features = ["MntCoffee" , "MntMeatProducts" , "Income" , "Kidhome" , "MntSweetProducts"]

chosen_feature_matrix, train_weights, bias = get_weights_and_bias(chosen_features)

y_pred = predict_output(X_test[chosen_features], train_weights, bias)

print ("R2 score: " , get_r2_score(y_pred, y_test))
print ("RMSE score: " , get_root_mean_square_error(y_pred, y_test))

R2 score:  0.5506047301545819
RMSE score:  5.12207803259083


median_value = df["NumPurchases"].median()
df.loc[df["NumPurchases"] >= median_value, "PurchaseRate"] = 1
df.loc[df["NumPurchases"] < median_value, "PurchaseRate"] = 0


def make_confusion_matrix(real_labels , pred_labels):
    cm = confusion_matrix(real_labels, pred_labels)
    # Create a heatmap
    sns.set(font_scale=1.4) 
    plt.figure(figsize=(5, 3))
    sns.heatmap(cm, annot=True, fmt="d", cmap = "RdYlGn", cbar=False)
    plt.xlabel("Predicted Labels")
    plt.ylabel("True Labels")
    plt.title("Confusion Matrix")
    plt.show()
    return cm


from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    df.drop(columns=["ID" , "PurchaseRate" , "NumPurchases" , "Dt_Customer"]),  # features
    df['PurchaseRate'],                 # target
    test_size=0.2,               # 80/20 split, a quarter of data used for testing
    random_state=84              # Seed for random number generator for reproducibility
    #,stratify=df['PurchaseRate']         # Stratify split based on labels, if classification problem
)
label_encoder = LabelEncoder()
y_train_encoded = label_encoder.fit_transform(y_train)


from sklearn.tree import DecisionTreeClassifier
decisionTree = DecisionTreeClassifier(
    criterion='entropy', 
    #splitter='random',   
    #max_depth=3,        
    min_samples_split=3, 
    min_samples_leaf=3,  
    random_state=42       
)
decisionTree.fit(X_train, y_train_encoded)
preds_DT = decisionTree.predict(X_test)


cm_DT = make_confusion_matrix(label_encoder.fit_transform(y_test) , preds_DT)


def scoring(cm):
    col_sum = [cm[0][i] + cm[1][i]  for i in range(2)]
    precision = [cm[i][i]/sum(cm[i]) for i in range(0,2)]
    recalls = [cm[i][i] / col_sum[i] for i in range(2)]
    F1_score = [(2*precision[i]*recalls[i])/ (precision[i] + recalls[i]) for i in range(2)]
    accuracy = sum(cm[i][i] for i in range(2))/sum(sum(cm))
    plt.clf()
    plt.figure(figsize=(5, 3))
    plt.scatter([0,1], precision)
    plt.scatter([0,1], recalls)
    plt.scatter([0,1], F1_score)
    plt.grid()
    plt.xlabel("class")
    plt.ylabel("metric")
    blue_patch = mpatches.Patch(color='blue',label='precision')
    orange_patch = mpatches.Patch(color='orange',label='F1_score')
    green_patch = mpatches.Patch(color='green',label='recalls')
    plt.legend(handles=[blue_patch,orange_patch,green_patch])

    # Adding the exact values on the plot
    for i in range(2):
        plt.text(i, precision[i], f'{precision[i]:.2f}', ha='center', va='bottom')
        plt.text(i, recalls[i], f'{recalls[i]:.2f}', ha='center', va='bottom')
        plt.text(i, F1_score[i], f'{F1_score[i]:.2f}', ha='center', va='bottom')
    plt.xticks([0, 1])
    plt.show()


    print(f"Precision: {precision[0]:.2f}")
    print(f"Recall: {recalls[0]:.2f}")
    print(f"F1 Score: {F1_score[0]:.2f}")
    print()

    weighted_average = sum(F1_score)/2
    micro_average = (sum(cm[i][i] for i in range(2)) / sum(sum(cm))) + (sum(cm[i][i] for i in range(2)) / sum(sum(cm))) / 2 
    macro_average = sum(F1_score)/2
    print("micro average: " + str(micro_average))
    print("weighted_average: " + str(weighted_average))
    print("macro average: " + str(macro_average))
    print("accuract: " + str(accuracy))


scoring(cm_DT)

<Figure size 640x480 with 0 Axes>

Precision: 0.87
Recall: 0.89
F1 Score: 0.88

micro average: 1.3348314606741574
weighted_average: 0.8894499622289482
macro average: 0.8894499622289482
accuract: 0.8898876404494382


from sklearn.model_selection import GridSearchCV
DecisionTreeGridSearch = GridSearchCV(
    estimator=DecisionTreeClassifier(),
    param_grid={
        "criterion": ["gini", "entropy"],
        "splitter": ["best", "random"],
        "max_depth": range(2, 20),
        "min_samples_split": range(2, 20),
        "min_samples_leaf": range(2, 20),
        "random_state": [84],
    },
    scoring="accuracy",
    cv= 4,
    n_jobs=-1,
)

DecisionTreeGridSearch.fit(X_train, y_train_encoded)
print(f"Best Parameters are : {DecisionTreeGridSearch.best_params_}")

Best Parameters are : {'criterion': 'entropy', 'max_depth': 4, 'min_samples_leaf': 17, 'min_samples_split': 2, 'random_state': 84, 'splitter': 'best'}


preds_DT_Grid = DecisionTreeGridSearch.predict(X_test)


cm_DT_Grid = make_confusion_matrix(label_encoder.fit_transform(y_test) , preds_DT_Grid)


scoring(cm_DT_Grid)

<Figure size 640x480 with 0 Axes>

Precision: 0.87
Recall: 0.93
F1 Score: 0.90

micro average: 1.3584269662921349
weighted_average: 0.9049609470344155
macro average: 0.9049609470344155
accuract: 0.9056179775280899


from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors=5 , metric = "manhattan" , weights = "distance")
knn.fit(X_train.to_numpy(), y_train_encoded)

KNeighborsClassifier(metric='manhattan', weights='distance')

KNeighborsClassifier(metric='manhattan', weights='distance')


KNN_preds = knn.predict(X_test.to_numpy())


cm_KNN = make_confusion_matrix(label_encoder.fit_transform(y_test) , KNN_preds)


scoring(cm_KNN)

<Figure size 640x480 with 0 Axes>

Precision: 0.88
Recall: 0.83
F1 Score: 0.85

micro average: 1.2842696629213481
weighted_average: 0.8560918425094997
macro average: 0.8560918425094997
accuract: 0.8561797752808988


import warnings
warnings.filterwarnings("ignore")
knn_grid = GridSearchCV(
    estimator=KNeighborsClassifier(),
    param_grid = {
    'n_neighbors': range(2, 10),
    'weights': ['uniform', 'distance'],
    'metric': ['euclidean', 'manhattan']
    },
    scoring="accuracy",
    cv=5,
    n_jobs=-1,
)
knn_grid.fit(X_train.to_numpy(), y_train_encoded)
print(f"Best Parameters : {knn_grid.best_params_}")

Best Parameters : {'metric': 'manhattan', 'n_neighbors': 5, 'weights': 'distance'}


KNN_grid_preds = knn_grid.predict(X_test.to_numpy())
cm_KNN_grid = make_confusion_matrix(label_encoder.fit_transform(y_test) , KNN_grid_preds)
scoring(cm_KNN_grid)

<Figure size 640x480 with 0 Axes>

Precision: 0.88
Recall: 0.83
F1 Score: 0.85

micro average: 1.2842696629213481
weighted_average: 0.8560918425094997
macro average: 0.8560918425094997
accuract: 0.8561797752808988


from sklearn.linear_model import LogisticRegression
log_reg = LogisticRegression(
    penalty='l2',  
    C=1.0,  
    solver='lbfgs',  
    max_iter=100, 
    multi_class='auto', 
    class_weight='balanced',  
    # l1_ratio=None  # Used only if penalty is 'elasticnet'
)
log_reg.fit(X_train, y_train_encoded)

LogisticRegression(class_weight='balanced')

LogisticRegression(class_weight='balanced')


Log_preds = log_reg.predict(X_test)
Log_cm = make_confusion_matrix(label_encoder.fit_transform(y_test) , Log_preds)
scoring(Log_cm)

<Figure size 640x480 with 0 Axes>

Precision: 0.92
Recall: 0.85
F1 Score: 0.88

micro average: 1.3280898876404494
weighted_average: 0.8853724196798811
macro average: 0.8853724196798811
accuract: 0.8853932584269663


param_grid = {
    'penalty': ['l1', 'l2', 'elasticnet', 'none'], 
    'C': np.logspace(-4, 4, 20),  
    'solver': ['liblinear', 'newton-cg', 'lbfgs', 'sag', 'saga'],  
    'max_iter': [50, 100, 200],  
    'class_weight': [None, 'balanced'],  
    'l1_ratio': np.linspace(0, 1, 10)  # The Elastic-Net mixing parameter (0 for L2 penalty, 1 for L1 penalty)
}

logreg_cv = GridSearchCV( LogisticRegression(), param_grid, cv=5, verbose=1, n_jobs=-1)  # Using all CPU cores with n_jobs=-1
logreg_cv.fit(X_train, y_train_encoded) 

print("Best parameters found: ", logreg_cv.best_params_)

Fitting 5 folds for each of 24000 candidates, totalling 120000 fits
Best parameters found:  {'C': 0.615848211066026, 'class_weight': None, 'l1_ratio': 0.0, 'max_iter': 50, 'penalty': 'l1', 'solver': 'liblinear'}


Log_preds_cv = log_reg.predict(X_test)
Log_cm_cv = make_confusion_matrix(label_encoder.fit_transform(y_test) , Log_preds_cv)
scoring(Log_cm_cv)

<Figure size 640x480 with 0 Axes>

Precision: 0.92
Recall: 0.85
F1 Score: 0.88

micro average: 1.3280898876404494
weighted_average: 0.8853724196798811
macro average: 0.8853724196798811
accuract: 0.8853932584269663


print(f"Decision Tree Accuracy for train datas: {decisionTree.score(X_train, y_train_encoded) * 100:5.2f}%")
print(f"Decision Tree Accuracy for test datas: {decisionTree.score(X_test, label_encoder.fit_transform(y_test)) * 100:5.2f}%")

print(f"KNN Accuracy for train datas: {knn.score(X_train.to_numpy(), y_train_encoded) * 100:5.2f}%")
print(f"KNN Accuracy for test datas: {knn.score(X_test.to_numpy(), label_encoder.fit_transform(y_test)) * 100:5.2f}%")

print(f"Logistic Regression Accuracy for train datas: {log_reg.score(X_train, y_train_encoded) * 100:5.2f}%")
print(f"Logistic Regression Accuracy for test datas: {log_reg.score(X_test, label_encoder.fit_transform(y_test) ) * 100:5.2f}%")

Decision Tree Accuracy for train datas: 98.43%
Decision Tree Accuracy for test datas: 88.99%
KNN Accuracy for train datas: 100.00%
KNN Accuracy for test datas: 85.62%
Logistic Regression Accuracy for train datas: 91.90%
Logistic Regression Accuracy for test datas: 88.54%


from sklearn.tree import plot_tree
plt.figure(figsize=(30, 10))
plot_tree(
    DecisionTreeGridSearch.best_estimator_,
    filled=True,
    feature_names = X_train.columns.tolist(),
)
plt.show()


plt.figure(figsize=(30, 10))
plot_tree(
    decisionTree,
    filled=True,
    feature_names = X_train.columns.tolist(),
)
plt.show()


from sklearn.ensemble import RandomForestClassifier

plt.figure(figsize=(7, 4))

test = []
train = []
for depth in range(5,50):
    RFC = RandomForestClassifier(max_depth=depth, random_state=84)
    RFC.fit(X_train.to_numpy(), y_train_encoded)
    train.append(RFC.score(X_train.to_numpy(), y_train_encoded))
    test.append(RFC.score(X_test.to_numpy(), label_encoder.fit_transform(y_test)))
plt.plot(range(5,50), train, "b", label="Train Accuracy")
plt.plot(range(5,50), test, "r", label="Test Accuracy")
plt.legend()
plt.xlabel("max_depths")
plt.ylabel("Accuracy")
plt.show()


from sklearn.ensemble import RandomForestClassifier

plt.figure(figsize=(7, 4))

test = []
train = []
for depth in range(5,50):
    RFC = RandomForestClassifier(n_estimators=depth, random_state=84)
    RFC.fit(X_train.to_numpy(), y_train_encoded)
    train.append(RFC.score(X_train.to_numpy(), y_train_encoded))
    test.append(RFC.score(X_test.to_numpy(), label_encoder.fit_transform(y_test)))
plt.plot(range(5,50), train, "b", label="Train Accuracy")
plt.plot(range(5,50), test, "r", label="Test Accuracy")
plt.legend()
plt.xlabel("n_estimators")
plt.ylabel("Accuracy")
plt.show()


RFC = RandomForestClassifier(n_estimators=15,max_depth= 7 , random_state=84)
RFC.fit(X_train.to_numpy(), y_train_encoded)

RandomForestClassifier(max_depth=7, n_estimators=15, random_state=84)

RandomForestClassifier(max_depth=7, n_estimators=15, random_state=84)


RFC_pred = RFC.predict(X_test.to_numpy())
RFC_cm_cv = make_confusion_matrix(label_encoder.fit_transform(y_test) , RFC_pred)
scoring(RFC_cm_cv)

<Figure size 640x480 with 0 Axes>

Precision: 0.88
Recall: 0.94
F1 Score: 0.91

micro average: 1.3719101123595505
weighted_average: 0.9140122854120901
macro average: 0.9140122854120901
accuract: 0.9146067415730337


print(f"Random Forest Accuracy for train datas: {RFC.score(X_train, y_train_encoded) * 100:5.2f}%")
print(f"Random Forest Accuracy for test datas: {RFC.score(X_test, label_encoder.fit_transform(y_test)) * 100:5.2f}%")

Random Forest Accuracy for train datas: 97.47%
Random Forest Accuracy for test datas: 91.46%


def addNoise(data, epsilon=1.0,sensitivity=1.0):
    scale = sensitivity / epsilon
    noise = np.random.laplace(0, scale=scale, size=data.shape)
    return data + noise


xTrainsNoisy = []
xTestsNoisy = []
sensitivityRange = [0.1, 0.5, 1.0, 2.0]
for sensitivity in sensitivityRange:
    xTrainsNoisy.append(addNoise(X_train.to_numpy(), sensitivity=sensitivity))
    xTestsNoisy.append(addNoise(X_test.to_numpy(), sensitivity=sensitivity))


len(xTrainsNoisy)

4


decisionTree_noisy = DecisionTreeClassifier(
    criterion='entropy', 
    #splitter='random',   
    #max_depth=3,        
    min_samples_split=3, 
    min_samples_leaf=3,  
    random_state=42       
)
decisionTree_noisy.fit(xTrainsNoisy[2], y_train_encoded)
preds_DT_noisy = decisionTree.predict(xTestsNoisy[2])
cm_DT_noisy = make_confusion_matrix(label_encoder.fit_transform(y_test) , preds_DT_noisy)
scoring(cm_DT_noisy)

<Figure size 640x480 with 0 Axes>

Precision: 0.25
Recall: 0.52
F1 Score: 0.34

micro average: 0.802247191011236
weighted_average: 0.48995299029352
macro average: 0.48995299029352
accuract: 0.5348314606741573


knn_noisy = KNeighborsClassifier(n_neighbors=5 , metric = "manhattan" , weights = "distance")
knn_noisy.fit(xTrainsNoisy[2], y_train_encoded)
KNN_preds_noisy = knn.predict(xTestsNoisy[2])
cm_KNN_noisy = make_confusion_matrix(label_encoder.fit_transform(y_test) , KNN_preds_noisy)
scoring(cm_KNN_noisy)

<Figure size 640x480 with 0 Axes>

Precision: 0.39
Recall: 0.55
F1 Score: 0.46

micro average: 0.8359550561797753
weighted_average: 0.5417431168681817
macro average: 0.5417431168681817
accuract: 0.5573033707865168


log_reg_noisy = LogisticRegression(
    penalty='l2',  
    C=1.0,  
    solver='lbfgs',  
    max_iter=100, 
    multi_class='auto', 
    class_weight='balanced',  
    # l1_ratio=None  # Used only if penalty is 'elasticnet'
)
log_reg_noisy.fit(xTrainsNoisy[2], y_train_encoded)
Log_preds_noisy = log_reg.predict(xTestsNoisy[2])
Log_cm_noisy = make_confusion_matrix(label_encoder.fit_transform(y_test) , Log_preds_noisy)
scoring(Log_cm_noisy)

<Figure size 640x480 with 0 Axes>

Precision: 0.53
Recall: 0.56
F1 Score: 0.54

micro average: 0.8662921348314607
weighted_average: 0.5748983739837399
macro average: 0.5748983739837399
accuract: 0.5775280898876405


print(f"Decision Tree Accuracy for train datas: {decisionTree_noisy.score(xTrainsNoisy[2], y_train_encoded) * 100:5.2f}%")
print(f"Decision Tree Accuracy for test datas: {decisionTree_noisy.score(xTestsNoisy[2], label_encoder.fit_transform(y_test)) * 100:5.2f}%")

print(f"KNN Accuracy for train datas: {knn_noisy.score(xTrainsNoisy[2], y_train_encoded) * 100:5.2f}%")
print(f"KNN Accuracy for test datas: {knn_noisy.score(xTestsNoisy[2], label_encoder.fit_transform(y_test)) * 100:5.2f}%")

print(f"Logistic Regression Accuracy for train datas: {log_reg_noisy.score(xTrainsNoisy[2], y_train_encoded) * 100:5.2f}%")
print(f"Logistic Regression Accuracy for test datas: {log_reg_noisy.score(xTestsNoisy[2], label_encoder.fit_transform(y_test) ) * 100:5.2f}%")

Decision Tree Accuracy for train datas: 95.28%
Decision Tree Accuracy for test datas: 49.44%
KNN Accuracy for train datas: 100.00%
KNN Accuracy for test datas: 52.13%
Logistic Regression Accuracy for train datas: 59.62%
Logistic Regression Accuracy for test datas: 60.22%


from xgboost import XGBClassifier

xgb = XGBClassifier(use_label_encoder=False)

param_grid = {
    'max_depth': [3, 5, 7, 9],
    'min_child_weight': [1, 3, 5],
    'gamma': [0, 0.1, 0.2],
    'subsample': [0.6, 0.8, 1.0],
    'colsample_bytree': [0.6, 0.8, 1.0],
    'learning_rate': [0.01, 0.1, 0.2]
}

grid_search = GridSearchCV(estimator=xgb, param_grid=param_grid, cv=3, scoring='accuracy', verbose=1, n_jobs=-1)

grid_search.fit(X_train.to_numpy(), y_train_encoded)

print(f"Best parameters found: {grid_search.best_params_}")
print(f"Best accuracy found: {grid_search.best_score_}")

best_estimator = grid_search.best_estimator_
XGB_preds = best_estimator.predict(X_test.to_numpy())
XGB_cm = make_confusion_matrix(label_encoder.fit_transform(y_test) , XGB_preds)
scoring(XGB_cm)

Fitting 3 folds for each of 972 candidates, totalling 2916 fits
Best parameters found: {'colsample_bytree': 0.6, 'gamma': 0.2, 'learning_rate': 0.1, 'max_depth': 7, 'min_child_weight': 1, 'subsample': 0.8}
Best accuracy found: 0.9566963675311061

<Figure size 640x480 with 0 Axes>

Precision: 0.88
Recall: 0.93
F1 Score: 0.91

micro average: 1.3719101123595505
weighted_average: 0.9140752032520325
macro average: 0.9140752032520325
accuract: 0.9146067415730337

	ID	Year_Birth	Income	Kidhome	Teenhome	Recency	MntCoffee	MntFruits	MntMeatProducts	MntFishProducts	MntSweetProducts	MntGoldProds	NumWebVisitsMonth	Complain	NumPurchases	UsedCampaignOffer
count	2240.000000	2240.000000	2017.000000	2240.000000	2240.000000	2240.000000	2035.000000	2240.000000	2240.000000	2240.000000	2240.000000	2227.000000	2040.000000	2240.000000	2240.000000	2240.000000
mean	5592.159821	1968.805804	52297.080317	0.437946	0.506250	49.109375	304.239312	26.302232	166.950000	37.525446	27.062946	43.847777	5.326961	0.009375	14.862054	0.271875
std	3246.662198	11.984069	25543.108215	0.563666	0.544538	28.962453	337.515534	39.773434	225.715373	54.628979	41.280498	51.897098	2.439349	0.096391	7.677173	0.445025
min	0.000000	1893.000000	2447.000000	-5.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
25%	2828.250000	1959.000000	35340.000000	0.000000	0.000000	24.000000	23.000000	1.000000	16.000000	3.000000	1.000000	9.000000	3.000000	0.000000	8.000000	0.000000
50%	5458.500000	1970.000000	51369.000000	0.000000	0.000000	49.000000	177.000000	8.000000	67.000000	12.000000	8.000000	24.000000	6.000000	0.000000	15.000000	0.000000
75%	8427.750000	1977.000000	68316.000000	1.000000	1.000000	74.000000	505.000000	33.000000	232.000000	50.000000	33.000000	56.000000	7.000000	0.000000	21.000000	1.000000
max	11191.000000	1996.000000	666666.000000	2.000000	2.000000	99.000000	1493.000000	199.000000	1725.000000	259.000000	263.000000	362.000000	20.000000	1.000000	44.000000	1.000000

	index	0
0	ID	2240
1	Year_Birth	59
2	Education	5
3	Marital_Status	8
4	Income	1810
5	Kidhome	5
6	Teenhome	3
7	Dt_Customer	663
8	Recency	100
9	MntCoffee	747
10	MntFruits	158
11	MntMeatProducts	558
12	MntFishProducts	182
13	MntSweetProducts	177
14	MntGoldProds	212
15	NumWebVisitsMonth	15
16	Complain	2
17	NumPurchases	39
18	UsedCampaignOffer	2

Course: AI

Project4: ML

Mohammad Amanlou- sid:810100084

Artificial Intelligence Course - Fall 1402¶

Computer Assignment #4 - Machine Learning¶

Project Summary¶

Part1.¶

Check the data set¶

Q1. Get the general data storage with the info and describe methods.¶

Q2. For each feature, obtain the number and proportion of missing data in Table¶

Q3.Draw a diagram of the relationship between the 6 and Ξ characteristics. Which features are more related to the target column?¶

Q4.For the features obtained in the previous step, draw a graph of the number of observations of each unique value¶

Q5.Check the relationship of the features with the target column more closely; You can use scatter and hexbin charts¶

Q.6.You can find and analyze any other review that helps you to know the collection¶

Part2.¶

PreProcess of dataset¶

Q.7.Two methods to solve the problem, Missing Values, deleting the entire column and matching empty values ​​with statistics¶

1. Deleting Columns with Missing Values¶

2. Imputation with Statistical Measures (Mean, Median, Mode)¶

3. Imputation Using Model-based Methods:¶

4. Dropping Rows with Missing Values:¶

Q8. Based on the results of the previous phase, which data have the most amount of missing data? Solve the missing data problem for all features using the mentioned methods¶

Q9.What is the purpose of normalizing or standardizing in numerical characteristics? Is there a need to do this work in this project?¶

Q10. To use the caregorical features, which are usually stored as a string or object in the data set, what methods are useful in teaching the model? Are all this features need to this methods¶

Q11.Is it possible to delete some columns? Why?¶

Q12.To train and finally evaluate the machine learning model, it is necessary to divide the data into two classes, test and train. What is the ratio of this part? What methods are there for dividing and making these two parts?¶

Q13. Sometimes, in addition to the above two categories, there is also a second category. Explain about this process (validation).¶

Q14.How is the K-Fold Cross Validation method performed? explain.¶

Part3.¶

Training, evaluation and regulation¶

phase1. Linear Regression¶

Q15.Explain the reason for obtaining the mentioned values:¶

Q16.In your opinion, which feature gives us a more accurate output than other features? Explain the reason for your choice¶

Q17.Read about RMSE, MSE, RSS and R2 score methods and explain each one in your report.¶

Q18. Using the RMSE method and R2 score, evaluate the predicted value. Do the above operation on several other features as well. What do you make of the obtained values?¶

phase2.Multiple Regression¶

Predicting output given regression weights¶

Computing the Derivative¶

Gradient Descent¶

Q18.1. Would selecting different initial weights make any difference?¶

Q18.2. Why are the features normalized?¶

Two Feature Regression¶

Three Feature Regression¶

Five Feature Regression¶

Part4.Classification¶

phase1. DT, KNN, LR¶

Q19. Obtain the accuracy of each model based on the drawn confusion matrix and explain the result¶

Decision Tree¶

Q20.Use GridSearchCV to find hyperparameters for models¶

KNN¶

Logistic Regression¶

Q21.Learn about underfitting and overfitting. Did these phenomena occur in your models?¶

Q22.Try to change some of the things you have done. Check their effect on the accuracy of their models. Note that the number of neighbors is sufficient for the KNN mode¶

Q23. Draw your final decision tree using the appropriate libraries¶

Phase2 .Ensemble Learning¶

Random Forest¶

Phase3. Differential Privacy¶

Q26. One of the common methods to anonymize the data set and maintain the personal security of people is to add new information to the data. What do you mean by adding new entries to the data in different rows of a dataset? Does it affect the security and privacy of people?¶

Q27. Among the conventional methods for adding noise to images, Laplace noise and linear noise can be mentioned. What is the difference between these two methods?¶

Phase4. Bonus: methods based on gradient boosting¶

Q29.What is Gradient-boosting ? What is the difference between Boosting Tree and Decision Tree?¶

Q30.Explain how XG-boost works.¶

Q31. Run XG-boost.¶

Q.7.Two methods to solve the problem, Missing Values, deleting the entire column and matching empty values with statistics¶